import math
import re

import scrapy
from ..items import HouesItem


class HouseSpider(scrapy.Spider):
    name = "house"
    allowed_domains = ["shangye.lianjia.com"]
    start_urls = [
        "https://shangye.lianjia.com/api/c/house/list?platform=2&device=1&page=1&city_id=510100&size=20&big_bizcircle_id=&district_id=510104&subway_id=&resblock_id=&bigbizcircle_switch=1&business_type=2"]
    types = ['写字楼买卖','写字楼租赁','商铺买卖','商铺租赁']

    def parse(self, response):
        """
        解析起始请求，构建循环，提交每页数据的抓取任务
        :param response:
        :return:
        """
        # print(response.json())
        data = response.json()
        pages = math.ceil(data['data']['total'] / 20)  # 向上取整
        pages = 50 if pages >= 50 else pages
        print(pages)

        # response.request.url获取响应的请求地址
        # 用正则去匹配需要进行修改的参数 将page=数字 改成page=%d 就可以随意改变page的数目了
        temp_url =re.sub(r'page=\d+', 'page=%d', response.request.url)
        # print(temp_url % 1)
        for i in range(pages):
            # print(temp_url % i)
            yield scrapy.Request(
                url = temp_url % i,
                callback=self.parse_data
            )

    def parse_data(self, response):
        """
        对每一页数据解析，交给pipline存储
        :param response:
        :return:
        """
        # print(response.json())
        type_i = int(re.findall(r'business_type=(\d+)',response.request.url)[0]) - 1
        houses = response.json()['data']['docs']
        for house in houses:
            item = HouesItem()
            # print(house)
            item['type'] = self.types[type_i]
            item['title'] = house['title'],
            item['area'] = house['area'],
            item['direction_desc'] = house['direction_desc'],
            item['fitment_name'] = house['fitment_name'],
            item['resblock_name'] = house['resblock_name'],
            item['unit_month_rent_price'] = house['unit_month_rent_price'],
            item['sell_price'] = house['sell_price'],
            item['geo'] = house['geo']
            # print(item)
            yield item
